package com.gary.spider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Spider {
	public static void main(String args[]){
		ArrayList<String> urllist=new ArrayList<String>();
		urllist.add("http://intranet.bleum.com/web/Pages/FrontEndPages/Defaultwithaward.aspx");
		Spider.getPage(urllist);
	}
	private static String trim(String s, int width) {
        if (s.length() > width)
            return s.substring(0, width-1) + ".";
        else
            return s;
    }

	public static List<Document> getPage(ArrayList<String> urllist){
		List<Document> doclist=new ArrayList<Document>();
		for(String url:urllist){
			Document doc = null;
			try {
				doc = Jsoup.connect(url).get();
			} catch (IOException e) {
				e.printStackTrace();
			}
			doclist.add(doc);
			Elements links = doc.select("a[href]");
			for (Element link : links) {
	            String l=String.format(" * a: <%s>  (%s)", link.attr("abs:href"),trim(link.text(), 35));
	        }

		}
		return doclist;
	}
}
